import numpy as np
import matplotlib.pyplot as plt
import os
import torchvision
import torch
import math
import time
import seaborn as sns
import pandas as pd
from matplotlib.colors import ListedColormap
import matplotlib

def Print_config(config):

    for key, value in config.items():
        print('Key: {}, Value: {}'.format(key, value))

class Plot:
    """
    the summary figure plot class for ploting the inner or the final experiment result.
    """
    def __init__(self):

        self.root_path = './img'
        self.path = os.path.join(self.root_path, 'Experiment')
        if os.path.exists(self.path) == False:
            os.makedirs(self.path)
            
    def plot_image(self, image, name):

        def imshow(img):
            np_img = img.cpu().numpy()
            plt.imshow(np.transpose(np_img, (1, 2, 0)))
        imshow(torchvision.utils.make_grid(image))
        plt.savefig(os.path.join(self.path, 'figure_{}.jpg'.format(name)))
        plt.close()
        print('Figure save done')

    def plot_UAPs(self, perturbations):
        """
        the input shape, we hope be: perturbatios = (batch_size, channels, width, height)
        """
        # perturbations = (perturbations + epsilon) / 2 * epsilon ## scale (-epsilon, epsilon) into (0, 1)
        # perturbations += epsilon
        # perturbations /= (2 * epsilon)
        sns.set_style('white')
        perturbations = torch.sign(perturbations)
        perturbations = ((perturbations + 1.0) / 2.0)
        perturbations = perturbations[:, 0, :, :].unsqueeze(dim=1)
        ## fig = plt.figure(figsize=(8, 4))
        size = perturbations.shape[0]

        matplotlib.rc('xtick', labelsize=8)
        matplotlib.rc('ytick', labelsize=8)

        orange = np.ones((256, 4))
        orange[:, 0] = np.linspace(255 / 256, 1, 256)  ## R = 244, 255, 255, 
        orange[:, 1] = np.linspace(153 / 256, 1, 256)  ## G = 185,128, 153, 
        orange[:, 2] = np.linspace(51 / 256, 1, 256) ## B = 96, 0, 51, 
        orange_cmp = ListedColormap(orange)

        blue = np.ones((256, 4))
        blue[:, 0] = np.linspace(51 / 256, 1, 256)  ## R = 124, 0, 51, 
        blue[:, 1] = np.linspace(153 / 256, 1, 256)  ## G = 174, 128, 153, 
        blue[:, 2] = np.linspace(255 / 256, 1, 256) ## B = 251, 255, 255,
        blue_cmp = ListedColormap(blue)

        newcolors = np.vstack((orange_cmp(np.linspace(0, 1, 128)),
                       blue_cmp(np.linspace(0, 1, 128))))# create a new colormaps with a name of OrangeBlue
        orange_blue = ListedColormap(newcolors, name='OrangeBlue')

        fig, ax = plt.subplots(nrows=3, ncols=3, figsize=(9, 9), sharex=True, sharey=True)

        for index, axi in enumerate(ax.flat):
            ## ax = plt.subplot(2, size // 2, index + 1, sharex=True, sharey=True)
            np_img = perturbations[index].cpu().numpy()
            axi.imshow(np.transpose(np_img, (1, 2, 0)), cmap=orange_blue)
            axi.axis('off')

            axi.set_title('MUAPs Index ' + '{}'.format(index), fontsize=16)

            axi.xaxis.set_major_locator(plt.MaxNLocator(6))
            axi.yaxis.set_major_locator(plt.MaxNLocator(6))

        plt.subplots_adjust(wspace=0.2, hspace=0.3)
        plt.savefig('{}'.format(os.path.join(self.path, 'MUAPs.pdf')), dpi=1200, bbox_inches='tight', pad_inches=0.3)
        print('Save done')
        plt.close()

    def plot_variant(self, ):

        """
        绘制变体实验的图片,图片结构如下,以CIFAR-10为例 
        Unatrget
            --> VGG16
                --> ASR
                    --> MPack
                    --> GA
                    --> SMPack
                --> AvgQue
                --> MedQue
            --> ResNet18
                --> ASR
                --> AvgQue
                --> MedQue
        Target
            --> VGG16
                --> ASR
                --> AvgQue
                --> MedQue
            --> ResNet18
                --> ASR
                --> AvgQue
                --> MedQue
        以Untarget/Target为划分,生成一张2*3的pdf图片
        """

        dicter = {
            'Untarget': {
                'ASR': {
                    'VGG16': {
                        'variant': np.concatenate([['Our Proposal' for i in range(3)], ['Variant' for i in range(3)]]),
                        'method': ['MPack', 'GA', 'SMPack', 'MPack', 'GA', 'SMPack',],
                        'value': [0.688, 0.998, 0.999, 0.612, 0.960, 0.963],
                        'variance': [0.018, 0.0, 0.0, 0.015, 0.006, 0.003],
                    },
                    'ResNet18': {
                        'variant': np.concatenate([['Our Proposal' for i in range(3)], ['Variant' for i in range(3)]]),
                        'method': ['MPack', 'GA', 'SMPack', 'MPack', 'GA', 'SMPack',],
                        'value': [0.897, 0.997, 1.0, 0.856, 0.990, 0.998],
                        'variance': [0.016, 0.002, 0.0, 0.019, 0.001, 0.002],
                    }
                },
                'AvgQue': {
                    'VGG16': {
                        'variant': np.concatenate([['Our Proposal' for i in range(3)], ['Variant' for i in range(3)]]),
                        'method': ['MPack', 'GA', 'SMPack', 'MPack', 'GA', 'SMPack',],
                        'value': [10.0, 600.2, 267.2, 10.0, 1319.9, 936.3],
                        'variance': [0.0, 34.6, 13.6, 0.0, 149.5, 55.6],
                    },
                    'ResNet18': {
                        'variant': np.concatenate([['Our Proposal' for i in range(3)], ['Variant' for i in range(3)]]),
                        'method': ['MPack', 'GA', 'SMPack', 'MPack', 'GA', 'SMPack',],
                        'value': [10.0, 345.0, 54.6, 10.0, 646.2, 161.8],
                        'variance': [0.0, 33.9, 14.3, 0.0, 57.2, 21.3],
                    }
                },
                'MedQue': {
                    'VGG16': {
                        'variant': np.concatenate([['Our Proposal' for i in range(3)], ['Variant' for i in range(3)]]),
                        'method': ['MPack', 'GA', 'SMPack', 'MPack', 'GA', 'SMPack',],
                        'value': [10.0, 200.0, 10.0, 10.0, 466.7, 10.0],
                        'variance': [0.0, 0.0, 0.0, 0.0, 62.4, 0.0],
                    },
                    'ResNet18': {
                        'variant': np.concatenate([['Our Proposal' for i in range(3)], ['Variant' for i in range(3)]]),
                        'method': ['MPack', 'GA', 'SMPack', 'MPack', 'GA', 'SMPack',],
                        'value': [10.0, 116.7, 10.0, 10.0, 350.0, 10.0],
                        'variance': [0.0, 23.6, 0.0, 0.0, 0.0, 0.0],
                    }
                },
            },
            'Target': {
                'ASR': {
                    'VGG16': {
                        'variant': np.concatenate([['Our Proposal' for i in range(3)], ['Variant' for i in range(3)]]),
                        'method': ['MPack', 'GA', 'SMPack', 'MPack', 'GA', 'SMPack',],
                        'value': [0.520, 0.937, 0.973, 0.222, 0.739, 0.795],
                        'variance': [0.031, 0.003, 0.007, 0.009, 0.010, 0.002],
                    },
                    'ResNet18': {
                        'variant': np.concatenate([['Our Proposal' for i in range(3)], ['Variant' for i in range(3)]]),
                        'method': ['MPack', 'GA', 'SMPack', 'MPack', 'GA', 'SMPack',],
                        'value': [0.660, 0.965, 0.993, 0.594, 0.869, 0.945],
                        'variance': [0.029, 0.004, 0.002, 0.016, 0.016, 0.013],
                    }
                },
                'AvgQue': {
                    'VGG16': {
                        'variant': np.concatenate([['Our Proposal' for i in range(3)], ['Variant' for i in range(3)]]),
                        'method': ['MPack', 'GA', 'SMPack', 'MPack', 'GA', 'SMPack',],
                        'value': [10.0, 2435.4, 1059.8, 10.0, 4180.9, 3412.7],
                        'variance': [0.0, 29.1, 89.1, 0.0, 116.5, 54.5],
                    },
                    'ResNet18': {
                        'variant': np.concatenate([['Our Proposal' for i in range(3)], ['Variant' for i in range(3)]]),
                        'method': ['MPack', 'GA', 'SMPack', 'MPack', 'GA', 'SMPack',],
                        'value': [10.0, 1693.1, 434.6, 10.0, 2802.7, 1310.2],
                        'variance': [0.0, 107.8, 40.4, 0.0, 156.3, 181.3],
                    }
                },
                'MedQue': {
                    'VGG16': {
                        'variant': np.concatenate([['Our Proposal' for i in range(3)], ['Variant' for i in range(3)]]),
                        'method': ['MPack', 'GA', 'SMPack', 'MPack', 'GA', 'SMPack',],
                        'value': [10.0, 1183.3, 93.3, 10.0, 1975.0, 1326.7],
                        'variance': [0.0, 23.6, 117.9, 0.0, 93.5, 77.3],
                    },
                    'ResNet18': {
                        'variant': np.concatenate([['Our Proposal' for i in range(3)], ['Variant' for i in range(3)]]),
                        'method': ['MPack', 'GA', 'SMPack', 'MPack', 'GA', 'SMPack',],
                        'value': [10.0, 900.0, 10.0, 10.0, 1250.0, 10.0],
                        'variance': [0.0, 0.0, 0.0, 0.0, 40.8, 0.0],
                    }
                },
            },
        }

        ## 
        sns.set_style('whitegrid')

        attack_methods = ['Untarget', 'Target']
        ## attack_methods = ['Untarget', ]
        datasets = ['VGG16', 'ResNet18']
        metrics = ['ASR', 'AvgQue', 'MedQue']

        colors = {'Our Proposal': 'red', 'Variant': 'blue'}

        for attack_method in attack_methods:

            fig = plt.figure(figsize=(12, 16))
            
            count = 1

            for metric in metrics:
                
                for dataset in datasets:
                    
                    ax = plt.subplot(3, 2, count)

                    sub_dicter = dicter[attack_method][metric][dataset]
                    data = pd.DataFrame(sub_dicter)

                    data['variant'] = pd.Categorical(data['variant'], data['variant'].unique())
                    data['method'] = pd.Categorical(data['method'], data['method'].unique())

                    data = data.sort_values(['variant', 'method'])

                    sns.barplot(data=data, x='method', y='value', hue='variant', ax=ax, width=0.6)

                    for container in ax.containers:
                        ax.bar_label(container)

                    x_coords = [p.get_x() + 0.5 * p.get_width() for p in ax.patches]
                    y_coords = [p.get_height() for p in ax.patches]
                    ax.errorbar(x=x_coords, y=y_coords, yerr=data['variance'], fmt="none", c="k", capsize=4, ecolor='lime')

                    if metric == 'ASR' and attack_method == 'Untarget':

                        ax.set_ylim(0.5, 1.05)

                    ax.set_title(dataset, fontsize=20)
                    ax.set_xlabel('Attack Method', fontsize=16)
                    ax.set_ylabel(metric, fontsize=16)
                    ax.xaxis.set_tick_params(labelsize=14)
                    plt.legend(fontsize=13, loc='upper left')
                    count += 1
            
            

            plt.subplots_adjust(wspace=0.2, hspace=0.3)
            plt.savefig(os.path.join(self.path, 'Varinat_{}.pdf'.format(attack_method)), dpi=800, bbox_inches='tight', pad_inches=0.3)
            print('Save Done')
            plt.close()

if __name__ == "__main__":

    ploter = Plot()
    ploter.plot_variant()

# class Experiment:
#     """
#     implement the experiment and responding result saving and showing.
#     """
#     def __init__(self, name, config):

#         self.root_path = './Result'
#         self.name = name
#         self.config = config
#         self.basic_path = os.path.join(self.root_path, self.name)
#         if os.path.exists(self.basic_path) == False:
#             os.makedirs(self.basic_path)
    
#     def generate_name(self, case='Case1'):

#         father_path = '{}/{}/{}/Batch({})_Size({})_Mode({})'.format(case, self.config['model_arch'], self.config['dataset'], self.config['base_batch'], self.config['base_size'], self.config['base_mode'])
#         path = os.path.join(self.basic_path, father_path, 'Eps({:.4f})_Norm({})_Method({})'.format(self.config['epsilon'], self.config['attck_norm'], self.config['attck_method']))
        
#         return path
    
#     def load_file(self, case='Case1'):

#         if os.path.exists(os.path.join(self.generate_name(case=case), 'result.pt')) == False:
#             raise Exception('Do not find result in path {}'.format(os.path.join(self.generate_name(case=case), 'result.pt')))

#         dicter = torch.load(os.path.join(self.generate_name(case=case), 'result.pt'))

#         return dicter

#     def save_total_file(self, case='Case1'):

#         if os.path.exists(os.path.join(self.basic_path, '{}'.format(case))) == False:
#             raise Exception('Did not have the following path <{}>'.format(os.path.join(self.basic_path, '{}'.format(case))))
        
#         if os.path.exists(os.path.join(self.basic_path, '{}'.format(case), 'data.pt')) == False:
#             dicter = {}
#         else:
#             dicter = torch.load(os.path.join(self.basic_path, '{}'.format(case), 'data.pt'))

#         dataset_name = self.config['dataset']
#         model_arch = self.config['model_arch']

#         if dataset_name not in dicter.keys():
#             dicter[dataset_name] = {}
        
#         sub_dicter = dicter[dataset_name]

#         context = self.load_file(case=case)

#         sub_dicter[model_arch] = context
    
#         torch.save(dicter, os.path.join(self.basic_path, '{}'.format(case), 'data.pt'))
    
#     def load_total_file(self, case='Case1'):

#         if os.path.exists(os.path.join(self.basic_path, '{}'.format(case))) == False:
#             raise Exception('Did not have the following path <{}>'.format(os.path.join(self.basic_path, '{}'.format(case))))
        
#         if os.path.exists(os.path.join(self.basic_path, '{}'.format(case), 'data.pt')) == False:
#             raise Exception('No such directory or file, please recheck.')
#         else:
#             dicter = torch.load(os.path.join(self.basic_path, '{}'.format(case), 'data.pt'))

#         return dicter
    
#     def latex_export(self, case='Case1'):

#         row_list = [
#             ['MNIST', 'CNN'], 
#             ['FMNIST', 'CNN'], 
#             ['CIFAR-10', 'VGG16'], 
#             ['CIFAR-10', 'ResNet18'],
#         ]

#         if case == 'Case1':
            
#             attack_method_list = ['OnePixel', 'SimBA', 'FNS', 'MPack', 'GA', 'SMPack', 'FGSM']
#             dicter = self.load_total_file(case='Case1')

#             print('CASE: 1')
#             ## print(dicter)

#             for row in row_list:
#                 print('Dataset: {}, Model: {}'.format(row[0], row[1]))
#                 str = ''
#                 sub_dicter = dicter[row[0]][row[1]]
#                 for attack_method in attack_method_list:
                    
#                     if attack_method in sub_dicter.keys():
#                         str += '&${:.1f}\\pm{:.1f}$ '.format(sub_dicter[attack_method][0] * 100, math.sqrt(sub_dicter[attack_method][1] * 100))
#                     else:
#                         str += '&- '
#                 str += '\\\\'
#                 print(str)
#                 print()
    
#         elif case == 'Case2':

#             attack_method_list = ['OnePixel', 'SimBA', 'FNS', 'MPack', 'GA', 'SMPack']
#             dicter = self.load_total_file(case='Case2')

#             print('CASE: 2')
#             ## print(dicter)

#             for row in row_list:
#                 print('Dataset: {}, Model: {}'.format(row[0], row[1]))
#                 avg_str = ''
#                 median_str = ''
#                 sub_dicter = dicter[row[0]][row[1]]
#                 for attack_method in attack_method_list:
                    
#                     if attack_method in sub_dicter.keys():
#                         avg_str += '&${:.1f}\\pm{:.1f}$ '.format(sub_dicter[attack_method]['Average'][0], math.sqrt(sub_dicter[attack_method]['Average'][1]))
#                         median_str += '&${:.1f}\\pm{:.1f}$ '.format(sub_dicter[attack_method]['Median'][0], math.sqrt(sub_dicter[attack_method]['Median'][1]))
#                     else:
#                         avg_str += '&- '
#                         median_str += '&- '         
                    
#                 avg_str += '\\\\'                                        
#                 median_str += '\\\\'
       
#                 print(avg_str)
#                 print()
#                 print(median_str)

#                 print('\n')
        
#         else:

#             raise ValueError

#     def show_result(self, case='Case1'):

#         print(self.load_file(case=case)) 
    
# class ExpMUAPs(Experiment):

#     def __init__(self, config, dataer, neter):

#         super(ExpMUAPs, self).__init__('MUAPs', config)
#         self.dataer = dataer
#         self.neter = neter

#     def attck_acc(self, atk, seed, repeat_times=3, default_num=500):

#         Attack_numbers_list = [0 for i in range(self.config['base_size'])]
        
#         loader = self.dataer.get_shuffle_loader(seed=seed, isTrain=False, batch_size=1)
        
#         count = 0
#         succ_number = 0
#         for (image, label) in loader:
            
#             if count == default_num:
#                 break
            
#             test_image = image.to(self.neter.device)
#             test_label = label.to(self.neter.device)
#             output = self.neter.net(test_image)
#             _, pre = torch.max(output.data, 1)
            
#             if pre != test_label:
#                 print('Pass the error sample')
#                 continue
            
#             succ_list = atk.untarget_UAP_show(
#                             origin_sample=image,
#                             source_class=label[0],
#                             epsilon=self.config['epsilon'],
#                             norm_type=self.config['attck_norm'],
#                             method=self.config['attck_method'],
#                             batch=self.config['base_batch'],
#                             size=self.config['base_size'],
#                             mode=self.config['base_mode'],
#                         )
#             count += 1
            
#             if len(succ_list) > 0:
#                 succ_number += 1
            
#             for index in succ_list:
#                 Attack_numbers_list[index] += 1
#         print(succ_number)
#         print(Attack_numbers_list)

# class ExpUntarget(Experiment):

#     def __init__(self, config, dataer, neter):

#         super(ExpUntarget, self).__init__('Untarget', config)
#         self.dataer = dataer
#         self.neter = neter
    
#     def latex_output(self, case='Case1'):
        
#         ## 转置成DFdata吧，然后存起来作为实验放图，然后再转它 

#         dicter = self.load_file(case=case)
#         if case == 'Case1':
#             list = ['OnePixel', 'SimBA', 'FNS', 'MPack', 'GA', 'SMPack', 'FGSM']
#             str = ''
#             for method in list:
#                 str += '&${:.1f}\\pm{:.1f}$ '.format(dicter[method][0] * 100, math.sqrt(dicter[method][1] * 100))
#             str += '\\\\'
#         elif case == 'Case2':
#             list = ['OnePixel', 'SimBA', 'FNS', 'MPack', 'GA', 'SMPack']
#             str = ''
#             for method in list:
#                 str += '&${:.1f}\\pm{:.1f}$ \n'.format(dicter[method]['Average'][0], math.sqrt(dicter[method]['Average'][1]))
#             str += '\n\n\n'
#             for method in list:
#                 str += '&${:.1f}\\pm{:.1f}$ \n'.format(dicter[method]['Median'][0], math.sqrt(dicter[method]['Median'][1]))
#             str += '\\\\'
#         else:
#             raise ValueError
        
#         return str

#     def Case_1_2(self, seed, repeat_times=3, default_num=500, way='SMPack', atk=None):

#         ASR_list = []
#         avg_Query_list = []
#         median_Query_list = []

#         for step in range(repeat_times):

#             loader = self.dataer.get_shuffle_loader(seed=seed+step, isTrain=False, batch_size=1)
#             succ_number = 0
#             query_buffer = 0
#             single_sample_query_list = []

#             count = 0
#             total = 0

#             for (image, label) in loader:
                
#                 if count == default_num:
#                     break
#                 total += 1
#                 test_image = image.to(self.neter.device)
#                 test_label = label.to(self.neter.device)
#                 output = self.neter.net(test_image)
#                 _, pre = torch.max(output.data, 1)
#                 if pre != test_label:
#                     print(pre)
#                     print(test_label)
#                     print('Pass the error sample')
#                     continue

#                 if way == 'SMPack':
#                     if atk == None:
#                         raise ValueError
#                     print('Epoch {}, Attacking {}-th sample'.format(step, count))
#                     flag, adv_image = atk.untarget_attack(
#                                         origin_sample=image,
#                                         source_class=label[0],
#                                         epsilon=self.config['epsilon'],
#                                         norm_type=self.config['attck_norm'],
#                                         method=self.config['attck_method'],
#                                         batch=self.config['base_batch'],
#                                         size=self.config['base_size'],
#                                         mode=self.config['base_mode'],
#                                     )
#                     if flag:
#                         succ_number += 1         
                
#                 elif way == 'GA':
#                     if atk == None:
#                         raise ValueError
#                     print('Epoch {}, Attacking {}-th sample'.format(step, count))
#                     flag, adv_image = atk.untarget_GA_attack(
#                                         origin_sample=image,
#                                         source_class=label[0],
#                                         epsilon=self.config['epsilon'],
#                                         norm_type=self.config['attck_norm'],
#                                         method=self.config['attck_method'],
#                                     )
#                     if flag:
#                         succ_number += 1     
                   
#                 elif way == 'MPack':
#                     if atk == None:
#                         raise ValueError
#                     print('Epoch {}, Attacking {}-th sample'.format(step, count))
#                     flag, adv_image = atk.untarget_MPack_attack(
#                                         origin_sample=image,
#                                         source_class=label[0],
#                                         epsilon=self.config['epsilon'],
#                                         norm_type=self.config['attck_norm'],
#                                         method=self.config['attck_method'],
#                                         batch=self.config['base_batch'],
#                                         size=self.config['base_size'],
#                                         mode=self.config['base_mode'],
#                                     )
#                     if flag:
#                         succ_number += 1   

#                 elif way == 'FNS':

#                     if atk == None:
#                         raise ValueError
#                     print('Epoch {}, Attacking {}-th sample'.format(step, count))
#                     flag, adv_image = atk.FNS_attack(
#                                         images=image,
#                                         labels=label,
#                                     )
#                     if flag:
#                         succ_number += 1 

#                 elif way == 'OnePixel':

#                     if atk == None:
#                         raise ValueError
#                     print('Epoch {}, Attacking {}-th sample'.format(step, count))
#                     adv_image = atk(image, label)

#                     adv_image = adv_image.to(self.neter.device)
#                     output = self.neter.net(adv_image)
#                     _, pre = torch.max(output.data, 1)

#                     if pre != test_label:                   
#                         succ_number += 1   

#                 elif way == 'SimBA':

#                     if atk == None:
#                         raise ValueError
#                     print('Epoch {}, Attacking {}-th sample'.format(step, count))
#                     flag, adv_image = atk.SimBA_attack(image, label[0], max_queries=5000, epsilon=self.config['epsilon'], targeted=False)

#                     if flag:
#                         succ_number += 1
                        
#                 elif way == 'FGSM':

#                     if atk == None:
#                         raise ValueError
#                     print('Epoch {}, Attacking {}-th sample'.format(step, count))
#                     adv_image = atk(image, label)

#                     adv_image = adv_image.to(self.neter.device)
#                     output = self.neter.net(adv_image)
#                     _, pre = torch.max(output.data, 1)

#                     if pre != test_label:                   
#                         succ_number += 1  

#                 elif way == 'ZOO':

#                     if atk == None:
#                         raise ValueError
#                     print('Epoch {}, Attacking {}-th sample'.format(step, count))
#                     adv_image, _ = atk.attack(image, label)
#                     adv_image = torch.from_numpy(adv_image).unsqueeze(0)

#                     adv_image = adv_image.to(self.neter.device)
#                     output = self.neter.net(adv_image)
#                     _, pre = torch.max(output.data, 1)

#                     if pre != test_label:                   
#                         succ_number += 1  

#                 elif way == 'Test':
#                     pass
#                 else:
#                     raise Exception('No such compared method.')
#                 print('Total : {}'.format(total))
#                 ## query counts
#                 print('='*50)
#                 print('Qurey for {}-th samples : {}'.format(count, self.neter.query - query_buffer))
#                 count += 1
#                 print('Total Query: {}'.format(self.neter.query))
#                 print('Last Total Query: {}'.format(query_buffer))
#                 print('='*50)
#                 single_sample_query_list.append(self.neter.query - query_buffer)
#                 query_buffer = self.neter.query

#             print('Way {} : The Epoch {}-th Untarget Attack ASR (Randomly sample {} samples) : {:.2f}%'.format(way, step, default_num, succ_number / default_num * 100))
            
#             ASR_list.append(succ_number / default_num) 
#             avg_Query_list.append(self.neter.query / default_num)
            
#             # median
#             single_sample_query_list.sort()
#             list_len = len(single_sample_query_list)
#             if list_len % 2 == 0:
#                 median_Query_list.append((single_sample_query_list[int(list_len / 2) - 1] + single_sample_query_list[int(list_len / 2)]) / 2)
#             else:
#                 median_Query_list.append(single_sample_query_list[int(list_len / 2)])

#             self.neter.clear_query()

#         case_1_path = self.generate_name(case='Case1')
#         case_2_path = self.generate_name(case='Case2')

#         # for case 1:
#         if os.path.exists(case_1_path) == False:
#             os.makedirs(case_1_path)
#         if os.path.exists(os.path.join(case_1_path, 'result.pt')) == False:
#             case1_dicter = {}
#             case1_dicter[way] = [np.mean(ASR_list), np.var(ASR_list)]
#         else:
#             case1_dicter = torch.load(os.path.join(case_1_path, 'result.pt'))
#             case1_dicter[way] = [np.mean(ASR_list), np.var(ASR_list)]
#         torch.save(case1_dicter, os.path.join(case_1_path, 'result.pt'))

#         # for case 2:
#         if os.path.exists(case_2_path) == False:
#             os.makedirs(case_2_path)
#         if os.path.exists(os.path.join(case_2_path, 'result.pt')) == False:
#             case2_dicter = {}
#             case2_dicter[way] = {
#                 'Average': [np.mean(avg_Query_list), np.var(avg_Query_list)],
#                 'Median': [np.mean(median_Query_list), np.var(median_Query_list)],
#                 }
#         else:
#             case2_dicter = torch.load(os.path.join(case_2_path, 'result.pt'))
#             case2_dicter[way] = {
#                 'Average': [np.mean(avg_Query_list), np.var(avg_Query_list)],
#                 'Median': [np.mean(median_Query_list), np.var(median_Query_list)],
#             }
#         torch.save(case2_dicter, os.path.join(case_2_path, 'result.pt'))


#     def Case_3(self, ):

#         pass

#     def Case_4(self, ):

#         pass

# class ExpTarget(Experiment):

#     def __init__(self, config, dataer, neter):

#         super(ExpTarget, self).__init__('Target', config)
#         self.dataer = dataer
#         self.neter = neter

#     def Case_1_2(self, seed, repeat_times=3, default_num=500, way='SMPack', atk=None):

#         ASR_list = []
#         avg_Query_list = []
#         median_Query_list = []

#         for step in range(repeat_times):

#             loader = self.dataer.get_shuffle_loader(seed=seed+step, isTrain=False, batch_size=1)
#             succ_number = 0
#             query_buffer = 0
#             single_sample_query_list = []

#             count = 0
#             total = 0
            
#             ##target attack mode: attacking the target class: [(label + 1) mod total_class_num] 
            

#             for (image, label) in loader:
                
#                 if count == default_num:
#                     break

#                 total += 1

#                 ## 确保该原始样本被正确分类
#                 test_image = image.to(self.neter.device)
#                 test_label = label.to(self.neter.device)
#                 output = self.neter.net(test_image)
#                 _, pre = torch.max(output.data, 1)
#                 if pre != test_label:
#                     print(pre)
#                     print(test_label)
#                     print('Pass the error sample')
#                     continue    
                
#                 ## implement of attacking the (label + 1) % total_class_num
#                 target_class = (label[0] + 1) % self.dataer.class_num

#                 if way == 'SMPack':
#                     if atk == None:
#                         raise ValueError
#                     print('Epoch {}, Attacking {}-th sample, source class: {}, target class: {}'.format(step, count, label[0], target_class))
#                     flag, adv_image = atk.target_attck(
#                                         origin_sample=image,
#                                         source_class=label[0],
#                                         target_class=target_class,
#                                         epsilon=self.config['epsilon'],
#                                         norm_type=self.config['attck_norm'],
#                                         method=self.config['attck_method'],
#                                         batch=self.config['base_batch'],
#                                         size=self.config['base_size'],
#                                         mode=self.config['base_mode'],
#                                     )
#                     if flag:
#                         succ_number += 1         
                
#                 elif way == 'GA':
#                     if atk == None:
#                         raise ValueError
#                     print('Epoch {}, Attacking {}-th sample, source class: {}, target class: {}'.format(step, count, label[0], target_class))
#                     flag, adv_image = atk.target_GA_attack(
#                                         origin_sample=image,
#                                         source_class=label[0],
#                                         target_class=target_class,
#                                         epsilon=self.config['epsilon'],
#                                         norm_type=self.config['attck_norm'],
#                                         method=self.config['attck_method'],
#                                     )
#                     if flag:
#                         succ_number += 1     
                   
#                 elif way == 'MPack':
#                     if atk == None:
#                         raise ValueError
#                     print('Epoch {}, Attacking {}-th sample, source class: {}, target class: {}'.format(step, count, label[0], target_class))
#                     flag, adv_image = atk.target_MPack_attack(
#                                         origin_sample=image,
#                                         source_class=label[0],
#                                         target_class=target_class,
#                                         epsilon=self.config['epsilon'],
#                                         norm_type=self.config['attck_norm'],
#                                         method=self.config['attck_method'],
#                                         batch=self.config['base_batch'],
#                                         size=self.config['base_size'],
#                                         mode=self.config['base_mode'],
#                                     )
#                     if flag:
#                         succ_number += 1   

#                 elif way == 'FNS':

#                     if atk == None:
#                         raise ValueError
#                     print('Epoch {}, Attacking {}-th sample, source class: {}, target class: {}'.format(step, count, label[0], target_class))
#                     atk.set_attack_mode(isTarget=True, class_index=target_class)
#                     flag, adv_image = atk.FNS_attack(
#                                         images=image,
#                                         labels=label,
#                                     )
#                     if flag:
#                         succ_number += 1 

#                 elif way == 'OnePixel':

#                     if atk == None:
#                         raise ValueError
#                     print('Epoch {}, Attacking {}-th sample, source class: {}, target class: {}'.format(step, count, label[0], target_class))
                                       
#                     # change into target attack mode
#                     atk.set_mode_targeted_by_function(target_map_function=lambda image, label: (label + 1) % self.dataer.class_num)
                    
#                     adv_image = atk(image, label)

#                     adv_image = adv_image.to(self.neter.device)
#                     output = self.neter.net(adv_image)
#                     _, pre = torch.max(output.data, 1)

#                     if pre[0] == target_class:                   
#                         succ_number += 1   

#                 elif way == 'SimBA':

#                     if atk == None:
#                         raise ValueError
#                     print('Epoch {}, Attacking {}-th sample, source class: {}, target class: {}'.format(step, count, label[0], target_class))
#                     flag, adv_image = atk.SimBA_attack(image, target_class, max_queries=10000, epsilon=self.config['epsilon'], targeted=True)

#                     if flag:
#                         succ_number += 1
                        
#                 elif way == 'FGSM':

#                     if atk == None:
#                         raise ValueError
#                     print('Epoch {}, Attacking {}-th sample, source class: {}, target class: {}'.format(step, count, label[0], target_class))
            
#                     # change into target attack mode
#                     atk.set_mode_targeted_by_function(target_map_function=lambda image, label: (label + 1) % self.dataer.class_num)
    
#                     adv_image = atk(image, label)

#                     adv_image = adv_image.to(self.neter.device)
#                     output = self.neter.net(adv_image)
#                     _, pre = torch.max(output.data, 1)

#                     if pre[0] == target_class:                   
#                         succ_number += 1  

#                 elif way == 'ZOO':

#                     if atk == None:
#                         raise ValueError
#                     print('Epoch {}, Attacking {}-th sample, source class: {}, target class: {}'.format(step, count, label[0], target_class))
#                     adv_image, _ = atk.attack(image, label)
#                     adv_image = torch.from_numpy(adv_image).unsqueeze(0)

#                     adv_image = adv_image.to(self.neter.device)
#                     output = self.neter.net(adv_image)
#                     _, pre = torch.max(output.data, 1)

#                     if pre != test_label:                   
#                         succ_number += 1  

#                 elif way == 'Test':
#                     pass

#                 else:
#                     raise Exception('No such compared method.')

#                 print('Total : {}'.format(total))
#                 ## query counts
#                 print('='*50)
#                 print('Qurey for {}-th samples : {}'.format(count, self.neter.query - query_buffer))
#                 count += 1
#                 print('Total Query: {}'.format(self.neter.query))
#                 print('Last Total Query: {}'.format(query_buffer))
#                 print('='*50)
#                 single_sample_query_list.append(self.neter.query - query_buffer)
#                 query_buffer = self.neter.query

#             print('Way {} : The Epoch {}-th Untarget Attack ASR (Randomly sample {} samples) : {:.2f}%'.format(way, step, default_num, succ_number / default_num * 100))
            
#             ASR_list.append(succ_number / default_num) 
#             avg_Query_list.append(self.neter.query / default_num)
            
#             # median
#             single_sample_query_list.sort()
#             list_len = len(single_sample_query_list)
#             if list_len % 2 == 0:
#                 median_Query_list.append((single_sample_query_list[int(list_len / 2) - 1] + single_sample_query_list[int(list_len / 2)]) / 2)
#             else:
#                 median_Query_list.append(single_sample_query_list[int(list_len / 2)])

#             self.neter.clear_query()

#         case_1_path = self.generate_name(case='Case1')
#         case_2_path = self.generate_name(case='Case2')

#         # for case 1:
#         if os.path.exists(case_1_path) == False:
#             os.makedirs(case_1_path)
#         if os.path.exists(os.path.join(case_1_path, 'result.pt')) == False:
#             case1_dicter = {}
#             case1_dicter[way] = [np.mean(ASR_list), np.var(ASR_list)]
#         else:
#             case1_dicter = torch.load(os.path.join(case_1_path, 'result.pt'))
#             case1_dicter[way] = [np.mean(ASR_list), np.var(ASR_list)]
#         torch.save(case1_dicter, os.path.join(case_1_path, 'result.pt'))

#         # for case 2:
#         if os.path.exists(case_2_path) == False:
#             os.makedirs(case_2_path)
#         if os.path.exists(os.path.join(case_2_path, 'result.pt')) == False:
#             case2_dicter = {}
#             case2_dicter[way] = {
#                 'Average': [np.mean(avg_Query_list), np.var(avg_Query_list)],
#                 'Median': [np.mean(median_Query_list), np.var(median_Query_list)],
#                 }
#         else:
#             case2_dicter = torch.load(os.path.join(case_2_path, 'result.pt'))
#             case2_dicter[way] = {
#                 'Average': [np.mean(avg_Query_list), np.var(avg_Query_list)],
#                 'Median': [np.mean(median_Query_list), np.var(median_Query_list)],
#             }
#         torch.save(case2_dicter, os.path.join(case_2_path, 'result.pt'))

#     def Case_3(self, ):

#         pass

#     def Case_4(self, ):

#         pass

# class ExpRobust(Experiment):

#     def __init__(self):
#         super(ExpRobust, self).__init__()

#     def Case_1_2(self, ):

#         pass

# class ExpExtension(Experiment):

#     def __init__(self):
#         super(ExpExtension, self).__init__()
    
#     def Case_variant(self, ):

#         pass

#     def Case_less_data(self, ):

#         pass

# class ExpParam(Experiment):

#     def __init__(self):
#         super(ExpParam, self).__init__()
    
#     def Case_batch(self, ):

#         pass

#     def Case_number(self, ):

#         pass

#     def Case_iter(self, ):

#         pass

#     def Case_pop(self, ):

#         pass




